import random
import redis
import requests

def get_url():
    client = redis.Redis(host='localhost', port=6379, db=0)
    url_bytes = client.lpop('已爬过的地址')
    url = url_bytes.decode()
    client.close()
    return url

def rollback_url(url):
    client = redis.Redis(host='localhost', port=6379, db=0)
    client.lpush('已爬过的地址', url)
    client.close()

def request(url):
    try:
        res = requests.get(url)
        print(res.json(), url)
    except Exception as e:
        print(f"请求地址{url}失败,错误原因{e}")
        rollback_url(url)

def main():
    for i in range(100):
        url = get_url()
        request(url)
        break_value = random.randint(1, 5)
        if break_value == 3:
            break
if __name__ == '__main__':
    main()




